{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp models.TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">This is a PyTorch implementation created by Ignacio Oguiza (oguiza@timeseriesAI.co) based on Sequencer: Deep LSTM for Image Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from tsai.imports import *\n",
    "from tsai.models.utils import *\n",
    "from tsai.models.layers import *\n",
    "from typing import Callable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoderLayer(nn.Module):\n",
    "    def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        self.bilstm = nn.LSTM(q_len, q_len, num_layers=1, bidirectional=True, bias=lstm_bias)\n",
    "        self.dropout = nn.Dropout(lstm_dropout) if lstm_dropout else nn.Identity()\n",
    "        self.fc = nn.Linear(2 * q_len, q_len)\n",
    "        self.lstm_norm = nn.LayerNorm(d_model)\n",
    "        self.pwff =  PositionwiseFeedForward(d_model, dropout=dropout, act=act, mlp_ratio=mlp_ratio)\n",
    "        self.ff_norm = nn.LayerNorm(d_model)\n",
    "        self.drop_path = DropPath(drop_path_rate) if drop_path_rate != 0 else nn.Identity()\n",
    "        self.pre_norm = pre_norm\n",
    "        self.transpose = Transpose(1,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.pre_norm:\n",
    "            x = self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(self.lstm_norm(x)))[0])))) + x\n",
    "            x = self.drop_path(self.pwff(self.ff_norm(x))) + x\n",
    "        else:\n",
    "            x = self.lstm_norm(self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(x))[0])))) + x)\n",
    "            x = self.ff_norm(self.drop_path(self.pwff(x)) + x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoder(nn.Module):\n",
    "    def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]\n",
    "        layers = []\n",
    "        for i in range(depth):\n",
    "            layer = _TSSequencerEncoderLayer(d_model, q_len=q_len, lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=dpr[i],\n",
    "                                      mlp_ratio=mlp_ratio, lstm_bias=lstm_bias, act=act, pre_norm=pre_norm)\n",
    "            layers.append(layer)\n",
    "        self.encoder = nn.Sequential(*layers)\n",
    "        self.norm = nn.LayerNorm(d_model) if pre_norm else nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.norm(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerBackbone(Module):\n",
    "    def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu',\n",
    "                 lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1,\n",
    "                 pre_norm:bool=False, use_token:bool=True,  use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None,\n",
    "                 cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None,\n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        if n_cat_embeds is not None:\n",
    "            n_cat_embeds = listify(n_cat_embeds)\n",
    "            if cat_embed_dims is None:\n",
    "                cat_embed_dims = [emb_sz_rule(s) for s in n_cat_embeds]\n",
    "            self.to_cat_embed = MultiEmbedding(c_in, n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos)\n",
    "            c_in, seq_len = output_size_calculator(self.to_cat_embed, c_in, seq_len)\n",
    "        else:\n",
    "            self.to_cat_embed = nn.Identity()\n",
    "\n",
    "        # Sequence embedding\n",
    "        if token_size is not None:\n",
    "            self.tokenizer = SeqTokenizer(c_in, d_model, token_size)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        elif tokenizer is not None:\n",
    "            if isinstance(tokenizer, nn.Module):  self.tokenizer = tokenizer\n",
    "            else: self.tokenizer = tokenizer(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        else:\n",
    "            self.tokenizer = nn.Identity()\n",
    "\n",
    "        # Feature extractor\n",
    "        if feature_extractor is not None:\n",
    "            if isinstance(feature_extractor, nn.Module):  self.feature_extractor = feature_extractor\n",
    "            else: self.feature_extractor = feature_extractor(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.feature_extractor, c_in, seq_len)\n",
    "        else:\n",
    "            self.feature_extractor = nn.Identity()\n",
    "\n",
    "        # Linear projection\n",
    "        if token_size is None and tokenizer is None and feature_extractor is None:\n",
    "            self.linear_proj = nn.Conv1d(c_in, d_model, 1)\n",
    "        else:\n",
    "            self.linear_proj = nn.Identity()\n",
    "\n",
    "        self.transpose = Transpose(1,2)\n",
    "\n",
    "        # Position embedding & token\n",
    "        if use_pe:\n",
    "            self.pos_embed = nn.Parameter(torch.zeros(1, seq_len, d_model))\n",
    "        self.use_pe = use_pe\n",
    "        self.cls_token = nn.Parameter(torch.zeros(1, 1, d_model))\n",
    "        self.use_token = use_token\n",
    "        self.emb_dropout = nn.Dropout(dropout) if dropout else nn.Identity()\n",
    "\n",
    "        # Encoder\n",
    "        self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias,\n",
    "                                         lstm_dropout=lstm_dropout, dropout=dropout,\n",
    "                                         mlp_ratio=mlp_ratio, drop_path_rate=drop_path_rate, act=act, pre_norm=pre_norm)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        x = self.to_cat_embed(x)\n",
    "\n",
    "        # Sequence embedding\n",
    "        x = self.tokenizer(x)\n",
    "\n",
    "        # Feature extractor\n",
    "        x = self.feature_extractor(x)\n",
    "\n",
    "        # Linear projection\n",
    "        x = self.linear_proj(x)\n",
    "\n",
    "        # Position embedding & token\n",
    "        x = self.transpose(x)\n",
    "        if self.use_pe:\n",
    "            x = x + self.pos_embed\n",
    "        if self.use_token: # token is concatenated after position embedding so that embedding can be learned using self.supervised learning\n",
    "            x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)\n",
    "        x = self.emb_dropout(x)\n",
    "\n",
    "        # Encoder\n",
    "        x = self.encoder(x)\n",
    "\n",
    "        # Output\n",
    "        x = x.transpose(1,2)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|exports\n",
    "class TSSequencerPlus(nn.Sequential):\n",
    "    r\"\"\"Time Series Sequencer model based on:\n",
    "\n",
    "    Tatsunami, Y., & Taki, M. (2022). Sequencer: Deep LSTM for Image Classification. arXiv preprint arXiv:2205.01972.\n",
    "    Official implementation: https://github.com/okojoalg/sequencer\n",
    "\n",
    "    Args:\n",
    "        c_in:               the number of features (aka variables, dimensions, channels) in the time series dataset.\n",
    "        c_out:              the number of target classes.\n",
    "        seq_len:            number of time steps in the time series.\n",
    "        d_model:            total dimension of the model (number of features created by the model).\n",
    "        depth:              number of blocks in the encoder.\n",
    "        act:                the activation function of positionwise feedforward layer.\n",
    "        lstm_dropout:       dropout rate applied to the lstm sublayer.\n",
    "        dropout:            dropout applied to to the embedded sequence steps after position embeddings have been added and\n",
    "                            to the mlp sublayer in the encoder.\n",
    "        drop_path_rate:     stochastic depth rate.\n",
    "        mlp_ratio:          ratio of mlp hidden dim to embedding dim.\n",
    "        lstm_bias:          determines whether bias is applied to the LSTM layer.\n",
    "        pre_norm:           if True normalization will be applied as the first step in the sublayers. Defaults to False.\n",
    "        use_token:          if True, the output will come from the transformed token. This is meant to be use in classification tasks.\n",
    "        use_pe:             flag to indicate if positional embedding is used.\n",
    "        n_cat_embeds:       list with the sizes of the dictionaries of embeddings (int).\n",
    "        cat_embed_dims:     list with the sizes of each embedding vector (int).\n",
    "        cat_padding_idxs:       If specified, the entries at cat_padding_idxs do not contribute to the gradient; therefore, the embedding vector at cat_padding_idxs\n",
    "                            are not updated during training. Use 0 for those categorical embeddings that may have #na# values. Otherwise, leave them as None.\n",
    "                            You can enter a combination for different embeddings (for example, [0, None, None]).\n",
    "        cat_pos:            list with the position of the categorical variables in the input.\n",
    "        token_size:         Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)\n",
    "        tokenizer:          nn.Module or callable that will be used to reduce the sequence length\n",
    "        feature_extractor:  nn.Module or callable that will be used to preprocess the time series before\n",
    "                            the embedding step. It is useful to extract features or resample the time series.\n",
    "        flatten:            flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.\n",
    "                            If use_token is False and flatten is False, the model will apply a pooling layer.\n",
    "        concat_pool:        if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.\n",
    "        fc_dropout:         dropout applied to the final fully connected layer.\n",
    "        use_bn:             flag that indicates if batchnorm will be applied to the head.\n",
    "        bias_init:          values used to initialized the output layer.\n",
    "        y_range:            range of possible y values (used in regression tasks).\n",
    "        custom_head:        custom head that will be applied to the network. It must contain all kwargs (pass a partial function)\n",
    "        verbose:            flag to control verbosity of the model.\n",
    "\n",
    "    Input:\n",
    "        x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',\n",
    "                 lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,\n",
    "                 pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,\n",
    "                 cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,\n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,\n",
    "                 flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,\n",
    "                 bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,\n",
    "                 **kwargs):\n",
    "\n",
    "        if use_token and c_out == 1:\n",
    "            use_token = False\n",
    "            pv(\"use_token set to False as c_out == 1\", verbose)\n",
    "        backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,\n",
    "                                      lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,\n",
    "                                      pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,\n",
    "                                      n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,\n",
    "                                      feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)\n",
    "\n",
    "        self.head_nf = d_model\n",
    "        self.c_out = c_out\n",
    "        self.seq_len = seq_len\n",
    "\n",
    "        # Head\n",
    "        if custom_head:\n",
    "            if isinstance(custom_head, nn.Module): head = custom_head\n",
    "            else: head = custom_head(self.head_nf, c_out, seq_len, **kwargs)\n",
    "        else:\n",
    "            nf = d_model\n",
    "            layers = []\n",
    "            if use_token:\n",
    "                layers += [TokenLayer()]\n",
    "            elif flatten:\n",
    "                layers += [Reshape(-1)]\n",
    "                nf = nf * seq_len\n",
    "            else:\n",
    "                if concat_pool: nf *= 2\n",
    "                layers = [GACP1d(1) if concat_pool else GAP1d(1)]\n",
    "            if use_bn: layers += [nn.BatchNorm1d(nf)]\n",
    "            if fc_dropout: layers += [nn.Dropout(fc_dropout)]\n",
    "\n",
    "            # Last layer\n",
    "            linear = nn.Linear(nf, c_out)\n",
    "            if bias_init is not None:\n",
    "                if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)\n",
    "                else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))\n",
    "            layers += [linear]\n",
    "\n",
    "            if y_range: layers += [SigmoidRange(*y_range)]\n",
    "            head = nn.Sequential(*layers)\n",
    "        super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))\n",
    "\n",
    "\n",
    "TSSequencer = TSSequencerPlus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=True)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=False)\n",
    "test_eq(model(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 1\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = 8.5\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor([bias_init]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature extractor\n",
    "\n",
    "It's a known fact that transformers cannot be directly applied to long sequences. To avoid this, we have included a way to subsample the sequence to generate a more manageable input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.data.validation import get_splits\n",
    "from tsai.data.core import get_ts_dls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZcAAABoCAYAAACNDM73AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGB1JREFUeJzt3QtUlHX+x/EvN0W5KV5QFMU1LPOaaKa5pqc0a7OkzLWs1Mouluwu2YVOgZZJZXnMC6XrJp3SspvGVrapm6J28W4XlbxLm6WVgliK4vzP9/dvaNAReICZhxner3PmDDPMPM93HnwEPnzn+wtwOBwOAQAAAAAAAADAgkArDwYAAAAAAAAAQBEuAwAAAAAAAAAsI1wGAAAAAAAAAFhGuAwAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgGWEywAAAAAAAAAAywiXAQAAAAAAAACWES4DAAB4SFZWlgQEBMjevXtL7uvXr5+5VLcJEyaYfbmKj4+XUaNGiafp69N96+t10v2Gh4eLt+j+9RgAAAAA8B7CZQAAgN999dVXMnToUGndurWEhoZKixYtZMCAATJjxgyP7fP77783oejmzZulJvjwww9rbEhbk2sDAAAAaqNguwsAAACoCT799FPp37+/tGrVSsaMGSPNmjWTvLw8+fzzz+WFF16QcePGVct+Pv7447PC5YkTJ5ou465du0p1ys3NlcDAQMsB7qxZsyyFuBrG//bbbxISElKJKqunNt1/cDA/2gIAAADexE/gAAAAIvLUU09JVFSUrFu3Tho0aFDqcwcPHqy2/dSpU0e8pW7duh7d/qlTp+T06dPmNWmnt53s3j8AAABQGzEWAwAAQER27dolHTp0OCtYVk2bNj1rvu/9998v8+fPl/PPP98Em4mJiZKTk1PuflxnLq9YsUJ69OhhPh49erTZ7pmzi91ZvXq1eZ7ut23btjJ79my3jztz5vLJkydNl3RCQoJ5bqNGjaRPnz6ydOlS83l9rHYGO1+j8+I6V/m5556TadOmmf1qeL1161a3M5eddu/eLVdeeaWEhYVJbGysPPHEE+JwOEo+r8dAn6vXrs7cZlm1Oe87s6N506ZNctVVV0lkZKSZ/3z55ZebTnR3c7HXrFkjKSkp0qRJE1NrUlKSHDp0qMyvAwAAAFDb0bkMAADw+2iHzz77TL7++mvp2LFjuY9fuXKlLFy4UJKTk03ImpmZKYMGDZK1a9dW6Pmqffv2JmxNS0uTu+66S/785z+b+3v37l3mXOiBAweaEFTDVO0eTk9Pl5iYmHL3p4/PyMiQO++8Uy6++GIpKCiQ9evXy8aNG81s6bvvvtuM6dCw+dVXX3W7jXnz5snx48dNvfq6o6OjTfeyO8XFxeaYXHLJJfLss8/KRx99ZGrVmvV1W1GR2lx988035nhqsPzQQw+ZkR0awmuwr1+7nj17lnq8jj1p2LChqU+DbQ3Q9Q8I+jUGAAAA4B7hMgAAgIiMHz/edLnq3GMNXjWY1E5XncPsbpawhtAazGrHsho+fLjpYtag+N13363QPjUQ1n3qc3r16iW33HJLuc/Rx2rn76pVq8x8aHXDDTdIp06dyn3uBx98IFdffbXMmTPH7ee1hnbt2pkA91y1fPfdd7Jz504TbjtpGOuOhtAaLk+fPt3cHjt2rAwePFieeeYZE8o3bty43Jqt1ObqscceM53a2uX9pz/9ydx32223ma+Rhs0aMLvSLm6dh+3shtbAXOvOz88341IAAAAAnI2xGAAAACKmc1c7l6+99lrZsmWL6bTVcQ4tWrSQ7Oxst2GnM1hWGvRed9118p///Md07HqCble3P2TIkJJg2dkBrbWWR0d+aEfvjh07Kl2DBtmuwXJ5tPv3zHEiRUVFsmzZMvEUPU4aFOtxcgbLqnnz5nLzzTebwFm7tl1pJ7brmA3944JuZ9++fR6rEwAAAPB1hMsAAAC/0znG2nV8+PBhM94iNTVVjh49KkOHDjWzhV3p3OIzaWftr7/+6rFZvbrd3377ze2+tSO3PDqK4siRI6ZO7XR+8MEH5csvv7RUQ5s2bSr82MDAwFLhrtJ9l9XtXF3HSb8O7o6JBvHalZyXl1fqftewXumIDKX/FgAAAAC4R7gMAABwhjp16pigefLkyfLiiy+a8QpvvfWW+Lq+ffuahQtffvllMxd67ty50q1bN3NdUfXq1avWmly7hV15qvv7XIKCgtze77r4IAAAAIDSCJcBAADK0L17d3N94MCBUve7Gy3x7bffSv369S2NjThXuOqOblfDXXf7zs3NrdA2dAG+0aNHy+uvv266dzt37mwW+qtMPeXRDuHdu3efdYxUfHx8qQ5h7ah25W4cRUVr0+OkXwd3x2T79u2mozouLs7CKwEAAADgDuEyAACAiHzyySduu1Q//PBDc33miAWdz7xx48aS2xrUvvfeezJw4MBzdsG6ExYW5jZcdUe3q7OVFy9eLPv37y+5f9u2bWYWc3l+/vnnUrfDw8PlvPPOkxMnTlSqnoqYOXNmycd6fPW2LpCoiyWq1q1bm9eVk5NT6nmZmZlnbauiten29OugXw/X8Rs//vijLFiwQPr06SORkZFVfm0AAABAbRdsdwEAAAA1wbhx48yc3qSkJLngggvMonOffvqpLFy40HTZarevKx0roUFvcnKy1K1btyQMnThxoqX9tm3b1iy099JLL0lERIQJUHv27HnO2ca6/Y8++sgsODd27Fg5deqUzJgxQzp06FDu/OQLL7xQ+vXrZxYi1A7m9evXy9tvv11q0T3nIoX6uvT1aVA7fPhwqYzQ0FBT68iRI81rWrJkiXzwwQfy6KOPlnR3R0VFyY033mheg3Ym6/F4//335eDBg2dtz0ptkyZNkqVLl5ogWY9TcHCwzJ492wTpulgjAAAAgKojXAYAABCR5557zsxV1k7lOXPmmHBZF3nTYPKxxx4zAbCryy67THr16mXCXu0i1uA2KyvLjJmwQrt4X3nlFbN44D333GPC4nnz5p0zXNbta5dySkqKpKWlScuWLU0NOrajvHBZQ9ns7Gz5+OOPTciqXcMawurCfk7XX3+9CdrfeOMNee2110y3cWXDZQ1/NVy+9957zT40PE9PTzd1u9JgWedaa8CuQf2wYcNkypQpJsB3ZaU2DdtXrVpljmtGRoYZ0aEBtz5PrwEAAABUXYCDVUoAAAAs0Q7b++67r9TIBwAAAACobZi5DAAAAAAAAACwjHAZAAAAAAAAAGAZ4TIAAAAAAAAAwDIW9AMAALCIJSsAAAAAgM5lAAAAAAAAAEAlEC4DAAAAAAAAAGr+WIzTp0/L999/LxERERIQEODt3QMAAAAAAAA+P6bt6NGjEhsbK4GB9I6iFoXLGizHxcV5e7cAAAAAAACAX8nLy5OWLVvaXQZqMa+Hy9qx/P/yRCTS27sHAAAAcIYuK/vaXQIAD9lyWY7dJQDwiAIRiXPJ2YBaEi7/MQpDg2XCZQAAAMBuQeFBdpcAwGP4vRvwZ4ychd0YygIAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAKDmz1wGAAAAAAAAAE8oLi6WkydP2l2GzwoKCpLg4OAKz/MmXAYAAAAAAADg8woLC+W7774Th8Nhdyk+rX79+tK8eXOpU6dOuY8lXAYAAAAAAADg8x3LGixrMNqkSZMKd97iDxrKFxUVyaFDh2TPnj2SkJAggYFlT1UmXAYAAAAAAADg03QUhoajGizXq1fP7nJ8lh67kJAQ2bdvnwmaQ0NDy3w8C/oBAAAAAAAA8At0LFdded3KpR5bDfsDAAAAAAAAANQyhMsAAAAAAAAAAMsIlwEAAAAAAADAT8THx8u0adO8si/CZQAAAAAAAAB+SUcwe/NidT50WZcJEyZIZaxbt07uuusuqZHhck5OjgwePFhiY2PNi1y8eLFnKgMAAAAAAAAAP3XgwIGSi3YaR0ZGlrpv/PjxJY91OBxy6tSpCm23SZMmUr9+famR4fKxY8ekS5cuMmvWLM9UBAAAAAAAAAB+rlmzZiWXqKgo08jrvL19+3aJiIiQJUuWSGJiotStW1dWr14tu3btkuuuu05iYmIkPDxcevToIcuWLStzLIZud+7cuZKUlGRC54SEBMnOzrYnXL7qqqtk0qRJphgAAAAAAAAAgGc88sgj8vTTT8u2bdukc+fOUlhYKFdffbUsX75cNm3aJIMGDTJTJvbv31/mdiZOnCjDhg2TL7/80jx/xIgR8ssvv9T8mcsnTpyQgoKCUhcAAAAAAAAAQNmeeOIJGTBggLRt21aio6PNRIm7775bOnbsaDqQn3zySfO58jqRR40aJTfddJOcd955MnnyZBNSr127Vmp8uJyRkWHaup2XuLg4T+8SAAAAAAAAAHxe9+7dS93WUFhnMbdv314aNGhgRmNoV3N5ncva9ewUFhZm5jsfPHiw5ofLqampkp+fX3LJy8vz9C4BAAAAAAAAwOeFhYWVuq3B8qJFi0z38apVq2Tz5s3SqVMnKSoqKnM7ISEhpW7rHObTp09Xub5g8TAdNq0XAAAAAAAAAEDlrVmzxoy4cK6Hp53Me/fuFbt4vHMZAAAAAAAAAFB1Omf53XffNR3LW7ZskZtvvrlaOpC91rmsafjOnTtLbu/Zs8e8GB0o3apVq+quDwAAAAAAAAAqxeEQvzJ16lS5/fbbpXfv3tK4cWN5+OGHpaCgwLZ6AhwOa4d4xYoV0r9//7PuHzlypGRlZZX7fH2xurCfSL6IRFqrFgAAAEC167Yh0e4SAHjIxsQNdpcAwCM0TIwy65vpwmwQOX78uGmCbdOmjYSGhtpdTq05lpY7l/v16ycW82gAAAAAAAAAgJ9h5jIAAAAAAAAAwDLCZQAAAAAAAACAZYTLAAAAAAAAAADLCJcBAAAAAAAAAJYRLgMAAAAAAAAALCNcBgAAAAAAAABYRrgMAAAAAAAAALCMcBkAAAAAAAAAYBnhMgAAAAAAAADAsmDrTwEAAAAAAACAmi9xY6JX97eh24YKPzYgIKDMz6enp8uECRMqVYdue9GiRTJkyBDxJMJlAAAAAAAAAPCyAwcOlHy8cOFCSUtLk9zc3JL7wsPDpabzerjscDh+/6jA27sGAAAA4EZxYbHdJQDwGH73Bvz53P4jZ4MvatasWcnHUVFRptvY9b65c+fK888/L3v27JH4+HhJTk6WsWPHms8VFRVJSkqKvPPOO3L48GGJiYmRe+65R1JTU81jVVJSkrlu3bq17N271z/C5Z9//vn3j+K8vWsAAAAAbmy5zO4KAHhOlN0FAPBwzqahJPzP/PnzTSfzzJkz5aKLLpJNmzbJmDFjJCwsTEaOHCnTp0+X7OxsefPNN6VVq1aSl5dnLmrdunXStGlTmTdvngwaNEiCgoI8VqfXw+Xo6GhzvX//fv7xA36moKBA4uLizH9mkZGRdpcDoBpxfgP+i/Mb8F+c34D/ys/PN4GiM2eD/0lPTzddy9dff7253aZNG9m6davMnj3bhMuarSYkJEifPn1Mx7N2Jzs1adLEXDdo0KBUJ7RfhMuBgYHmWoNlvrkB/knPbc5vwD9xfgP+i/Mb8F+c34D/cuZs8C/Hjh2TXbt2yR133GG6lZ1OnTpV0qw7atQoGTBggJx//vmmO/maa66RgQMHer1WFvQDAAAAAAAAgBqisLDQXP/zn/+Unj17lvqcc8RFt27dzCzmJUuWyLJly2TYsGFyxRVXyNtvv+3VWgmXAQAAAAAAAKCGiImJkdjYWNm9e7eMGDHinI/Td6X89a9/NZehQ4eaDuZffvnFjEsJCQmR4uJi/wuX69ata2aG6DUA/8L5Dfgvzm/Af3F+A/6L8xvwX5zf/m/ixImSnJxsxmBoaHzixAlZv369HD58WFJSUmTq1KnSvHlzs9ifjkd56623zHxlnbOs4uPjZfny5XLppZeafycNGzb0SJ0BDofD4ZEtAwAAAAAAAIAXHD9+3IyJ0IXvQkNDxddkZWXJ3//+dzly5EjJfQsWLJApU6aYhfzCwsKkU6dO5jFJSUlmZEZmZqbs2LHDjMro0aOHeayGzerf//63CaH37t0rLVq0MNeeOJaEywAAAAAAAAB8mq+Hy756LFlSEgAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgGWEywAAAAAAAACAmh0uz5o1S+Lj480g6J49e8ratWu9uXsAHpCRkWFWJI2IiJCmTZvKkCFDJDc31+6yAHjA008/LQEBAWZ1YgC+73//+5/ccsst0qhRI6lXr55ZfXz9+vV2lwWgioqLi+Xxxx83izDpud22bVt58sknxeFw2F0aAItycnJk8ODBEhsba34OX7x4canP63mdlpYmzZs3N+f7FVdcITt27JDajv/vvHsMvRYuL1y4UFJSUiQ9PV02btwoXbp0kSuvvFIOHjzorRIAeMDKlSvlvvvuk88//1yWLl0qJ0+elIEDB8qxY8fsLg1ANVq3bp3Mnj1bOnfubHcpAKrB4cOH5dJLL5WQkBBZsmSJbN26VZ5//nlp2LCh3aUBqKJnnnlGXnzxRZk5c6Zs27bN3H722WdlxowZdpcGwCL9vVrzM23WdEfP7enTp8tLL70kX3zxhYSFhZms7fjx41IbBQUFmeuioiK7S/F5v/76q7nWnxXLE+DwUpyvncra3ajf4NTp06clLi5Oxo0bJ4888og3SgDgBYcOHTIdzBo69+3b1+5yAFSDwsJC6datm2RmZsqkSZOka9euMm3aNLvLAlAF+vP3mjVrZNWqVXaXAqCaXXPNNRITEyP/+te/Su674YYbTFfja6+9ZmttACpPO5cXLVpk3i2sNM7TjuYHHnhAxo8fb+7Lz883539WVpYMHz5cahs9Jvv37zdNb3psAgOZBlyZY6jBsjYDN2jQwHTFlydYvED/YrBhwwZJTU0tuU+/wNqu/9lnn3mjBABeot/MVHR0tN2lAKgm+u6Ev/zlL+b7tobLAHxfdna26Wy68cYbzR+EW7RoIWPHjpUxY8bYXRqAKurdu7fMmTNHvv32W2nXrp1s2bJFVq9eLVOnTrW7NADVaM+ePfLDDz+Yn9GdoqKiTHOnZm21MVzWAF7DUD02+/bts7scn6bBcrNmzSr0WK+Eyz/99JOZ+6R/PXGlt7dv3+6NEgB4gb4jQWex6ttsO3bsaHc5AKrBG2+8YcZZ6VgMAP5j9+7d5m3zOrbu0UcfNed4cnKy1KlTR0aOHGl3eQCq+M6EgoICueCCC8xbxPV38aeeekpGjBhhd2kAqpEGy8pd1ub8XG2kP8skJCQwGqMKdBSGc8RIjQmXAdSe7savv/7adEYA8H15eXnyt7/9zcxT18V4AfjXH4S7d+8ukydPNrcvuugi8z1cZzYSLgO+7c0335T58+fLggULpEOHDrJ582bTAKJvEef8BlAb6LQEfn/xHq8MH2ncuLFJvH/88cdS9+vtirZYA6jZ7r//fnn//fflk08+kZYtW9pdDoBqoCOtdNaWzlsODg42F337vC4aoh9rJxQA36RvGb3wwgtL3de+fXszpxCAb3vwwQdN97K+Jb5Tp05y6623yj/+8Q/JyMiwuzQA1ciZp5G1oVaEy9qSnpiYKMuXLy/VLaG3e/Xq5Y0SAHhw2LsGy7qwwH//+19p06aN3SUBqCaXX365fPXVV6bjyXnRTkd9W61+bOWtUgBqFh1hlZubW+o+nc/aunVr22oCUD10IaYzF7HS79n6OzgA/6G/e2uI7Jq16UicL774gqwNXuW1sRg6z03fgqO/lF588cVmlfljx47J6NGjvVUCAA+NwtC33L333nsSERFRMttJFxLQFakB+C49p8+cnx4WFiaNGjVirjrg47SLURf90rEYw4YNk7Vr15oFwPQCwLcNHjzYzFhu1aqVGYuxadMms5jf7bffbndpACwqLCyUnTt3ltzWheq0ySM6Otqc4zryRhfc1hnDGjY//vjjZgTOkCFDbK0btUuAQ9sOvWTmzJkyZcoUEz517drVvK1WV7EE4Nursbozb948GTVqlNfrAeBZ/fr1M9/D9Y/EAHybjrNKTU2VHTt2mF9ItRlkzJgxdpcFoIqOHj1qAiZ9Z6GOt9Kg6aabbpK0tDTzrmIAvmPFihXSv3//s+7X5s2srCzzTuL09HTzx+EjR45Inz59JDMzU9q1a2dLvaidvBouAwAAAAAAAAD8g1dmLgMAAAAAAAAA/AvhMgAAAAAAAADAMsJlAAAAAAAAAIBlhMsAAAAAAAAAAMsIlwEAAAAAAAAAlhEuAwAAAAAAAAAsI1wGAAAAAAAAAFhGuAwAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgFj1fw7YwpaVgPHmAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1600x50 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TSTensor(samples:8, vars:3, len:5000, device=mps:0, dtype=torch.float32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.zeros((10, 3, 5000))\n",
    "y = np.random.randint(0,2,X.shape[0])\n",
    "splits = get_splits(y)\n",
    "dls = get_ts_dls(X, y, splits=splits)\n",
    "xb, yb = dls.train.one_batch()\n",
    "xb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you try to use SequencerPlus, it's likely you'll get an 'out-of-memory' error.\n",
    "\n",
    "To avoid this you can subsample the sequence reducing the input's length. This can be done in multiple ways. Here are a few examples: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 99])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separable convolution (to avoid mixing channels)\n",
    "feature_extractor = Conv1d(xb.shape[1], xb.shape[1], ks=100, stride=50, padding=0, groups=xb.shape[1]).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolution (if you want to mix channels or change number of channels)\n",
    "feature_extractor=MultiConv1d(xb.shape[1], 64, kss=[1,3,5,7,9], keep_original=True).to(default_device())\n",
    "test_eq(feature_extractor.to(xb.device)(xb).shape, (xb.shape[0], 64, xb.shape[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MaxPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.MaxPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AvgPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.AvgPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once you decide what type of transform you want to apply, you just need to pass the layer as the feature_extractor attribute:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 1000\n",
    "c_out = 2\n",
    "d_model = 128\n",
    "\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "feature_extractor = partial(Conv1d, ks=5, stride=3, padding=0, groups=xb.shape[1])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, d_model=d_model, feature_extractor=feature_extractor)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.utils import alphabet, ALPHABET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = alphabet[np.random.randint(0,3,40)]\n",
    "b = ALPHABET[np.random.randint(6,10,40)]\n",
    "c = np.random.rand(40).reshape(4,1,10)\n",
    "map_a = {k:v for v,k in enumerate(np.unique(a))}\n",
    "map_b = {k:v for v,k in enumerate(np.unique(b))}\n",
    "n_cat_embeds = [len(m.keys()) for m in [map_a, map_b]]\n",
    "szs = [emb_sz_rule(n) for n in n_cat_embeds]\n",
    "a = np.asarray(a.map(map_a)).reshape(4,1,10)\n",
    "b = np.asarray(b.map(map_b)).reshape(4,1,10)\n",
    "inp = torch.from_numpy(np.concatenate((c,a,b), 1)).float()\n",
    "feature_extractor = partial(Conv1d, ks=3, padding='same')\n",
    "model = TSSequencerPlus(3, 2, 10, d_model=64, cat_pos=[1,2], feature_extractor=feature_extractor)\n",
    "test_eq(model(inp).shape, (4,2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes you have a samples with a very long sequence length. In those cases you may want to reduce it's length before passing it to the transformer. To do that you may just pass a token_size like in this example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128, 168])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "SeqTokenizer(2, 128, 60)(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 5])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "model = TSSequencerPlus(2, 5, 10080, d_model=64, token_size=60)\n",
    "model(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/069_models.TSSequencerPlus.ipynb saved at 2025-01-20 10:26:55\n",
      "Correct notebook to script conversion! 😃\n",
      "Monday 20/01/25 10:26:58 CET\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
